package com.xiaotu.spider.project.pageprocessor.list;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import com.google.gson.Gson;
import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 BaiKeListPageProcessor
 * @日期 2017年8月24日
 * @作者 王艳龙
 * @功能 百度百科网站数据爬取
 */

@Component("BaiKeListPageProcessor")
@Scope("prototype")
public class BaiKeListPageProcessor extends AUrlPageProcessor {

	private String fileName; // 存放文件路径
	private String fileBaiKe; //存放读取出的百科内容
	private static final String BAIKE_TV_NAME = "网大名称";
	private static final String BAIKE_NAME = "中文名";
    private static final String BAIKE_COMPANY = "出品公司";
	private static final String BAIKE_DIRECT = "导    演";
	private static final String BAIKE_STAR 	 = "主    演";
	private static final String BAIKE_LENGTH = "片    长";
	private static final String BAIKE_TIME = "上映时间";
	private static final List<String> LABELS_BAIKE_ALL = Arrays.asList("网大名称", "中文名", "出品公司", "导    演","主    演", "片    长", "上映时间");
	
	@Override
	protected String[] getStartRequestUrls() {
		List<JSONObject> targetList = readFileColumn(fileName); //读取excel网大名称
        List<String> linkList = new ArrayList<String>();
        for (JSONObject target : targetList)
        {
            String tvName = target.getString("tvName");
            linkList.add("https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=0&rsv_idx=1&tn=baidu&wd="+KeywordEncode.encode(tvName)
					+"%20%E7%94%B5%E5%BD%B1%20%E7%99%BE%E5%BA%A6%E7%99%BE%E7%A7%91&name="+tvName);
            //linkList.add("https://baike.baidu.com/item/" + KeywordEncode.encode(tvName));
        }
        String[] urlArr = new String[linkList.size()];
        linkList.toArray(urlArr);
        return urlArr;
	}

	@Override
	public void initRunParams(Map<String, Object> map) {
		if (map.get("fileName") == null)
			throw new SpiderException(SpiderExceptionCode.PageProcessorParameterErrorException);
		fileName = map.get("fileName") + StringUtils.EMPTY;
		
		if (map.get("fileBaiKe") == null)
			throw new SpiderException(SpiderExceptionCode.PageProcessorParameterErrorException);
		fileBaiKe = map.get("fileBaiKe") + StringUtils.EMPTY;
	}

	@Override
	public String getPageDataType(Page page) {
		String url = page.getUrl().toString();
		/*if (url.contains("https://baike.baidu.com")) {
			return Constants.PageDataType.SEARCH_LIST;
		} else*/ 
		if(url.contains("https://www.baidu.com/s?")){
			return Constants.PageDataType.TV_COMMENT;
		} else if(url.contains("http://www.baidu.com/link")){
			return Constants.PageDataType.TV_PRO_COMMENT;
		}else {
			return Constants.PageDataType.TV_COMMENT;
		}

	}

	@Override
	public void process(Page page) {
		switch (this.getPageDataType(page)) {
		/*case Constants.PageDataType.SEARCH_LIST:
			this.searchProcess(page);
			break;*/
		case Constants.PageDataType.TV_COMMENT:
			this.lsitProcess(page);
			break;
		case Constants.PageDataType.TV_PRO_COMMENT:
			this.searchDeilProcess(page);
			break;
		default:
			break;
		}
	}
	
	public void lsitProcess(Page page){
		Map<String, Object> tvMap = page.getRequest().getExtras();
		String url = page.getUrl().toString();
		String tvName = url.substring(url.indexOf("name=") + 5, url.length());
		tvMap.put("tvName", tvName);
		String linkUrl = page.getHtml().xpath("//div[@class='result-op c-container xpath-log']/h3/a/@href").get();
		if(StringUtils.isNotEmpty(linkUrl)){
			PageUtils.addTargetRequest(page, linkUrl, page.getRequest().getExtras());
		}
	}
	
	//百度百科搜索查找
	public void searchProcess(Page page) {
		Map<String, Object> tvMap = page.getRequest().getExtras();
		String url = page.getUrl().toString();
		String tvName = url.substring(url.indexOf("item/") + 5, url.length());
		List<String> list1 =new ArrayList<String>();
		List<String> list2 = new ArrayList<String>();
		List<String> divListKey = page.getHtml().xpath("//div[@class='basic-info cmn-clearfix']/dl/dt").all();
		List<String> divListValue = page.getHtml().xpath("//div[@class='basic-info cmn-clearfix']/dl/dd").all();
		if(divListKey.size()==0 || divListValue.size()==0){
			String baikeUrl ="https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=0&rsv_idx=1&tn=baidu&wd="+tvName
							+"20%E7%94%B5%E5%BD%B1%20%E7%99%BE%E5%BA%A6%E7%99%BE%E7%A7%91";
			PageUtils.addTargetRequest(page, baikeUrl, page.getRequest().getExtras());
			
		}else{
			for (String ListKeyStr : divListKey) {
				Html html1 =new Html(ListKeyStr);
				String nameKey = html1.xpath("//dt[@class='basicInfo-item name']/text()").get().trim();
					list1.add(nameKey);
			}
			for (String ListValueStr : divListValue) {
				Html html2 =new Html(ListValueStr);
				String nameValue = html2.xpath("//dd[@class='basicInfo-item value']/allText()").get();
				list2.add(nameValue);
			}
			Map<String, String> map = new HashMap<String, String>();
			for (int i = 0; i < list1.size(); i++) {
				String movieValue =list2.get(i);
				String key = list1.get(i);
				if (key.contains(BAIKE_NAME) || key.contains(BAIKE_COMPANY) || key.contains(BAIKE_DIRECT) 
					|| key.contains(BAIKE_STAR) || key.contains(BAIKE_LENGTH) || key.contains(BAIKE_TIME)) {
					this.parseBaseinfo(movieValue, map, key,tvMap);
				}
				
			}
				StringBuffer buffer = new StringBuffer();
				for (String keys : LABELS_BAIKE_ALL)
					buffer.append(SepratorUtil.SEP_PIPE
							+ (map.containsKey(keys) ? map.get(keys)
									: StringUtils.EMPTY));
				FileUtil.writeFile(fileBaiKe, buffer.toString()
						+ SepratorUtil.LINE_SEPRATOR);
		}
	}
	
	//百度搜索精确匹配
	public void searchDeilProcess(Page page) {
		Map<String, Object> tvMap = page.getRequest().getExtras();
		List<String> list1 =new ArrayList<String>();
		List<String> list2 = new ArrayList<String>();
		List<String> divListKey = page.getHtml().xpath("//div[@class='basic-info cmn-clearfix']/dl/dt").all();
		List<String> divListValue = page.getHtml().xpath("//div[@class='basic-info cmn-clearfix']/dl/dd").all();
		for (String ListKeyStr : divListKey) {
			Html html1 =new Html(ListKeyStr);
			String nameKey = html1.xpath("//dt[@class='basicInfo-item name']/text()").get().trim();
			list1.add(nameKey);
		}
		for (String ListValueStr : divListValue) {
			Html html2 =new Html(ListValueStr);
			String nameValue = html2.xpath("//dd[@class='basicInfo-item value']/allText()").get();
			list2.add(nameValue);
		}
		Map<String, String> map = new HashMap<String, String>();
		for (int i = 0; i < list1.size(); i++) {
			String movieValue =list2.get(i);
			String key = list1.get(i);
			if (key.contains(BAIKE_NAME) || key.contains(BAIKE_COMPANY) || key.contains(BAIKE_DIRECT) 
				|| key.contains(BAIKE_STAR) || key.contains(BAIKE_LENGTH) || key.contains(BAIKE_TIME)) {
				this.parseBaseinfo(movieValue, map, key,tvMap);
			}
			
		}
			//网大名称
			StringBuffer buffer = new StringBuffer();
			map.put(BAIKE_TV_NAME,buffer.append(tvMap.get("tvName")).toString());
			for (String keys : LABELS_BAIKE_ALL)
				buffer.append(SepratorUtil.SEP_PIPE
						+ (map.containsKey(keys) ? map.get(keys)
								: StringUtils.EMPTY));
			FileUtil.writeFile(fileBaiKe, buffer.toString()
					+ SepratorUtil.LINE_SEPRATOR);
}

	/**
	 * TEXT文本基本信息解析
	 * @param html 页面元素
	 * @param map缓存
	 */
	private void parseBaseinfo(String movieVal, Map<String, String> map, String key,Map<String, Object> tvMap) {
		StringBuffer buffer = new StringBuffer();
		/*//网大名称
		map.put(BAIKE_TV_NAME,buffer.append(tvMap.get("tvName")).toString());*/
		//电影名称
		if(key.contains(BAIKE_NAME)){
			map.put(BAIKE_NAME,buffer.append(movieVal).toString());
		//出品公司
		}else if (key.contains(BAIKE_COMPANY)) {
			map.put(BAIKE_COMPANY,buffer.append(movieVal).toString());
		//导演
		}else if (key.contains(BAIKE_DIRECT)) {
			map.put(BAIKE_DIRECT,buffer.append(movieVal).toString());
		//主演
		} else if (key.contains(BAIKE_STAR)) {
			map.put(BAIKE_STAR,buffer.append(movieVal).toString());
		//时长
		} else if (key.contains(BAIKE_LENGTH)) {
			map.put(BAIKE_LENGTH,buffer.append(movieVal).toString());
		//上映时间
		} else if (key.contains(BAIKE_TIME)) {
			map.put(BAIKE_TIME,buffer.append(movieVal).toString());
		}
		
	}
	
	/**
     * 读取文件指定列
     *
     * @param fileName 文件名称
     * @return 行列表
     */
    public static List<JSONObject> readFileColumn(String fileName)
    {
        Map<String, Object> map = new HashMap<String, Object>();
        Workbook wb = null;
        String json;
        List<JSONObject> list = new ArrayList<JSONObject>();
        try
        {
            InputStream inputStream = new FileInputStream(fileName);
            if (fileName.endsWith("xls"))
            {
                wb = new HSSFWorkbook(inputStream); // 解析xls格式
            } else if (fileName.endsWith("xlsx"))
            {
                wb = new XSSFWorkbook(inputStream); // 解析xlsx格式
            }
            Sheet sheet = wb.getSheetAt(0); // 第一个工作表

            int firstRowNum = 1;
            int lastRowNum = sheet.getLastRowNum();

            Row row = null;
            Cell cell_a1 = null;
            for (int i = firstRowNum; i <= lastRowNum; i++)
            {
                row = sheet.getRow(i);          //取得第i行
                cell_a1 = row.getCell(0);      //取得i行的第一列
                String cellValue = cell_a1.getStringCellValue().trim();
                map.put("tvName", cellValue);
                json = new Gson().toJson(map);
                JSONObject jsStr = JSONObject.fromObject(json);
                list.add(jsStr);
            }

        }
        catch (FileNotFoundException e)
        {
            e.printStackTrace();
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        return list;
    }

	@Override
	protected String getSpiderType() {
		return "BaiKeList";
	}
}
